package cn.jdy.plugins.admin;

import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestMethod;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;

import cn.jdy.authority.entity.User;
import cn.jdy.common.constant.GlobalConstant;
import cn.jdy.common.ctrl.BaseCtrl;
import cn.jdy.framework.log.BaseLog;
import cn.jdy.framework.util.DateUtil;
import cn.jdy.framework.util.FileDownLoadThread;
import cn.jdy.framework.util.FileUtil;
import cn.jdy.framework.util.HttpUtil;
import cn.jdy.platform.entity.News;
import cn.jdy.platform.service.NewsService;
import cn.jdy.plugins.entity.RobotContent;
import cn.jdy.plugins.entity.RobotLinks;
import cn.jdy.plugins.entity.RobotManager;
import cn.jdy.plugins.service.RobotContentService;
import cn.jdy.plugins.service.RobotLinksService;
import cn.jdy.plugins.service.RobotManagerService;

/**
 * 采集点管理
 */
@Controller
@RequestMapping("/robotManager")
public class RobotManagerCtrl extends BaseCtrl {
    @Autowired
    private RobotManagerService robotManagerService;
    @Autowired
    private RobotLinksService robotLinksService;
    @Autowired
    private RobotContentService robotContentService;
    @Autowired
    private NewsService newsService;
    
    private static String PREFIX = "/upload/news";// 抓取的图片路径前缀
    
    /**
     * 跳转到采集列表页面
     * 
     * @return
     */
    @RequestMapping("/list")
    public ModelAndView list() {
        return new ModelAndView("/robotManager/robotManagerList.jsp");
    }
    
    /**
     * 查询采集列表
     * 
     * @param robotManager
     * @param fromDate
     * @param toDate
     * @return
     */
   @RequestMapping("/queryByPage")
   @ResponseBody
   public Map<String, Object> queryByPage(RobotManager robotManager, String fromDate, String toDate) {
       return robotManagerService.queryByPage(robotManager, fromDate, toDate);
   }
   
   @RequestMapping("/toAdd")
   public ModelAndView toAdd() {
	   ModelAndView mv = new ModelAndView("/robotManager/addRobotManager.jsp");
	   mv.addObject("column", this.confService.queryByType(GlobalConstant.TYPE_NEWS, 0L));
	   return mv;
   }
    
  /**
   * 添加采集点
   * 
   * @param robotManager
   * @return
   */
  @RequestMapping(value = "/add", method = RequestMethod.POST)
	@ResponseBody
	public String add(RobotManager robotManager, Long pid) {
    	String status = "200";
    	try {
			// 如果二级新闻类别为空，则使用一级类别
			if (null == robotManager.getColumnId()) {
				robotManager.setColumnId(pid);
			}
    		Long id = robotManager.getId();
    		if (id != null) {
    			robotManagerService.updateWithValidator(robotManager);
    		} else {
    			robotManager.setCreateTime(DateUtil.formatDate(new Date(), DateUtil._DEFAULT2));
        		robotManagerService.save(robotManager);
    		}
		} catch (Exception e) {
			status = "500";
			BaseLog.e(this.getClass(), "添加采集站点失败！", e);
			e.printStackTrace();
		}
    	
  	return status;
  }
  
  /**
   * 修改采集点
   * 
   * @param id
   * @return
   */
  @RequestMapping("/update/{id}")
  public ModelAndView update(@PathVariable Long id) {
  	ModelAndView mv = new ModelAndView("/robotManager/addRobotManager.jsp");
  	mv.addObject("column", this.confService.queryByType(GlobalConstant.TYPE_NEWS, 0L));
  	RobotManager robotManager = robotManagerService.find(id);
  	mv.addObject("robotManager", robotManager);
      return mv;
  }

  /**
   * 删除采集点
   * 
   * @param id
   * @return
   */
  @RequestMapping("/delete/{id}")
  public ModelAndView delete(@PathVariable Long id) {
  	robotManagerService.delete(id);
      return list();
  }
  
  /**
   * 采集列表网址
   * 
   * @param id
   * @return
   */
  @RequestMapping(value = "/robotURL", method = RequestMethod.POST)
  @ResponseBody
  public String robotURL(String id) {
  	String status = "200";
  	try {
  		RobotManager robotManager = robotManagerService.find(Long.parseLong(id));
  		if (robotManager != null) {
            // 抓取整个网页
            String url = robotManager.getUrl();
            String charset = robotManager.getCharset();
            String[] urls = url.split("\r\n");// 每行一条
            
            for (int i = 0; i < urls.length; i++) {
              BaseLog.d(this.getClass(), "抓取整个网页开始================================================================");
              String html = HttpUtil.sendByUserAgent(urls[i], charset, 18500);
              BaseLog.d(this.getClass(), "抓取整个网页结束================================================================");
              
              if (html !=null && !"".equals(html)) {
                // 获取文章列表区域内容
                String startFlag = robotManager.getStartflag();
                String endFlag = robotManager.getEndflag();
                BaseLog.d(this.getClass(), "抓取区域内容开始================================================================");
                String area = getAreaContent(html,startFlag,endFlag);
                BaseLog.d(this.getClass(), "抓取区域内容结束================================================================");
                
                if (area !=null && !"".equals(area)) {
                  BaseLog.d(this.getClass(), "抓取文章链接开始================================================================");
                  // 解析获取采集链接列表
                  List<Map<String, Object>> links = getLink(area, urls[i],robotManager.getContain());
                  if (links!=null && links.size()>0) {
                    for (Map<String, Object> link : links) {
                      String href = (String)link.get("href");// 获取文章链接地址
                      List<RobotLinks> list = robotLinksService.find("link", href);// 查询是否已经采集
                      if (list.isEmpty()) {// 判断是否已采集
                        RobotLinks robotlink = new RobotLinks();
                        robotlink.setManagerid(id);
                        robotlink.setTitle((String)link.get("title"));
                        robotlink.setCreateTime(DateUtil.getCurrentDate2());
                        robotlink.setLink(href);
                        robotlink.setStatus("0");
                        robotLinksService.save(robotlink);
                      }
                      BaseLog.d(this.getClass(), href);
                   }
                    // 更新最后 采集时间
                    robotManagerService.update(Long.valueOf(id), "lasttime", DateUtil.getCurrentDate2());
                    BaseLog.d(this.getClass(), "抓取超链接结束================================================================");
                  } else {
                    status = "404"; // 未抓取到链接数据
                  }
                } else {
                  status = "404"; // 未截取到指定内容
                }
              } else {
                status = "404"; // 未抓取到网页数据
              }
            }
  		}
	} catch (Exception e) {
		status = "500";
		BaseLog.e(this.getClass(), "采集网址失败！", e);
		e.printStackTrace();
	}
  	return status;
 }
  
  /**
   * 采集文章内容
   * 
   * @param id
   * @return
   */
  @RequestMapping(value = "/robotContent", method = RequestMethod.POST)
  @ResponseBody
  public String robotContent(String id) {
  	String status = "200";
  	try {
  		RobotManager robotManager = robotManagerService.find(Long.parseLong(id));
  		if (robotManager != null) {// 判断采集规则 是否为空
  			String charset = robotManager.getCharset();
  			String matcher_content = robotManager.getMatcher_content();
  			String array [] = matcher_content.split("\\[内容\\]");// 内容起止标示
  			String startFlag = array[0]; //起
  			String endFlag = array[1];   //止
  			
  			// 获取该采集规则下面的 采集目标列表
  			List<RobotLinks> list = robotLinksService.find("managerid", id, "status", "0");
  			if (list != null && list.size() > 0) {
  				for (RobotLinks link : list) {
          	// 解析超链接获取内容
            String content = getContent(link.getLink(), charset, startFlag, endFlag);
            if (!"".equals(content)) {
            	RobotContent robotcontent = new RobotContent();
            	robotcontent.setCreateTime(DateUtil.getCurrentDate2());
            	robotcontent.setLinkid(link.getId());
            	// 需要替换文章内容中的远程图片
            	content = watchImg(content);
            	robotcontent.setContent(content);
            	robotContentService.save(robotcontent);
            	
            	robotLinksService.update(link.getId(), "status", "1");
            }
          }
  			} else {
  				status = "404"; // 没有需要采集内容的数据 
  			}
  		} else {
  			status = "500";
  		}
  	}  catch (Exception e) {
		status = "500";
		BaseLog.e(this.getClass(), "采集内容失败！", e);
		e.printStackTrace();
	}
  	return status;
  }
  
  /**
   * 发布内容
   * 
   * @param id
   * @return
   */
  @RequestMapping(value = "/publishContent", method = RequestMethod.POST)
  @ResponseBody
  public String publishContent(String id) {
  	String status = "200";
  	try {
  		RobotManager robotManager = robotManagerService.find(Long.parseLong(id));
  		if (robotManager != null) {
      		List<RobotLinks> list = robotLinksService.find("managerid", id, "status", "1");
      		if (list!=null && list.size()>0) {
      			for (RobotLinks link : list) {
      				String linkid = link.getId();
      				RobotContent robotContent = robotContentService.findOne("linkid", linkid);
      				
      				// 新增新闻
      				News news = new News();
      				User user = getUser();
      				news.setCreateTime(DateUtil.getCurrentDate(DateUtil._DEFAULT2));
      				news.setUserid(user.getId());
      				news.setUsername(user.getUsername());
      				news.setAuthor(user.getUsername());
      				news.setTitle(link.getTitle());
      				news.setContext(robotContent.getContent());
      				news.setStatus("1");
      				news.setColumnId(robotManager.getColumnId());
      				newsService.save(news);
      				
      				// 修改采集链接状态为 [已发布]
      				robotLinksService.update(link.getId(), "status", "2");
      				
      				// 删除采集内容
      				robotContentService.delete("linkid", linkid);
      			}
      		} else {
      			status = "404"; // 没有需要发布内容的数据
      		}
  		} else {
  			status = "500";
  		}
	} catch (Exception e) {
		status = "500";
		BaseLog.e(this.getClass(), "发布内容失败！", e);
		e.printStackTrace();
	}
  	
  	return status;
  }
  
  /**
   * @return 获取指定区域的内容
   * <div class=\"partbox\">(.*?)<div class=\"con_right\">
   */
  public static String getAreaContent(String s,String startFlag,String endFlag) {
      // 获取指定区域的内容
      final Pattern pa = Pattern.compile(startFlag+"(.*?)"+endFlag, Pattern.DOTALL);  
      final Matcher ma = pa.matcher(s);  
      StringBuffer sb = new StringBuffer(); 
      
      while (ma.find()) {  
          sb.append(ma.group(1));
      }  
      return sb.toString();
  }
  
  /**
   * @return 获得网页标题
   */
  public static String getTitle(final String s, String matcher) {
      String regex;
      String title = "";
      final List<String> list = new ArrayList<String>();
      // regex = "<title>.*?</title>";
      regex = matcher.replace("[内容]", ".*?");
      final Pattern pa = Pattern.compile(regex, Pattern.CANON_EQ);
      final Matcher ma = pa.matcher(s);
      while (ma.find()) {
          list.add(ma.group());
      }
      for (int i = 0; i < list.size(); i++) {
          title = title + list.get(i);
      }
      return outTag(title);
  }
  
   
  /**
   * 解析出超链接
   * @param s 待解析的内容
   * @param url 
   * @param must
   * @return
   */
  public static List<Map<String, Object>> getLink(final String s, final String url,String must) {
      String regex;
      final List<Map<String, Object>> list = new ArrayList<Map<String, Object>>();
      regex = "<a[^>]*href=(\"([^\"]*)\"|\'([^\']*)\'|([^\\s>]*))[^>]*>(.*?)</a>";
      final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
      final Matcher ma = pa.matcher(s);
      while (ma.find()) {
      	Map<String, Object> linkMap = new HashMap<String, Object>();
          String href = ma.group(2);
          if(href.indexOf(must)==-1){//必须包含
            continue;// 跳出本次循环，继续下次循环
          }
          if(href.matches("(http|https)://.*")){
          	linkMap.put("href", href);
          	linkMap.put("title", filterHtml(ma.group(5)));
          	list.add(linkMap);
          } else {
          	Pattern p = Pattern.compile("(http://|https://)?([^/]*)",Pattern.CASE_INSENSITIVE);
          	Matcher matcher = p.matcher(url);
          	matcher.find();
          	if (href.indexOf("/") != 0) {
          		linkMap.put("href", matcher.group()+"/"+href);
          	} else {
          		linkMap.put("href", matcher.group()+href);
          	}
          	linkMap.put("title", filterHtml(ma.group(5)));
          	list.add(linkMap);
          }
//            BaseLog.d(RobotManagerCtrl.class, "整个链接==>"+ma.group());
//            BaseLog.d(RobotManagerCtrl.class, "链接地址==>"+ma.group(2)); 
//            BaseLog.d(RobotManagerCtrl.class, "链接标题==>"+ma.group(5));
      }
      return list;
  }
  
  /**
   * @return 去掉标记
   */
  public static String outTag(final String s) {
  	return s.replaceAll("<.*?>", "");
  }
  
  /**
   * @return 获得脚本代码
   */
  public static List<String> getScript(final String s) {
      String regex;
      final List<String> list = new ArrayList<String>();
      regex = "<script.*?</script>";
      final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
      final Matcher ma = pa.matcher(s);
      while (ma.find()) {
          list.add(ma.group());
      }
      return list;
  }
  
  /**
   * @return 获得CSS
   */
  public List<String> getCSS(final String s) {
      String regex;
      final List<String> list = new ArrayList<String>();
      regex = "<style.*?</style>";
      final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
      final Matcher ma = pa.matcher(s);
      while (ma.find()) {
          list.add(ma.group());
      }
      return list;
  }
  
  /**
   * 获取链接详细内容
   * @param link_url
   */
  public String getContent(String link_url, String charset, String startFlag, String endFlag) {
  	String content = "";
	    String CONTENT_REX = "(?i)(<iframe([^>]*)>([\\s\\S]*)</iframe>|"
                                          + "<script([^>]*)>([\\s\\S]*)</script>|"
                                          + "<style([^>]*)>([\\s\\S]*)</style>|"
                                          + "<div([^>]*)>|</div>|"
                                          + "<a([^>]*)>|</a>|"
                                          + "<!--([\\s\\S]*)-->)";
  	
      BaseLog.d(this.getClass(), "抓取链接详细内容开始================================================================");
      try {
          String html = HttpUtil.sendByUserAgent(link_url, charset, 18500);
          // BaseLog.d(this.getClass(), html);
          
          // startFlag = "<div class=\"art_con\" id=\"contentText\">";
          // endFlag = "<div class=\"art_page\" id=\"art_page\">";
         BaseLog.d(this.getClass(), "抓取区域内容开始================================================================");
         String area = getAreaContent(html,startFlag,endFlag);
         BaseLog.d(this.getClass(), area);
         BaseLog.d(this.getClass(), "抓取区域内容结束================================================================");
          
          // 过滤不需要的内容：例如 广告，脚本等
          content = area.replaceAll(CONTENT_REX, "");
          BaseLog.d(this.getClass(), content);
      } catch (Exception e) {
      	e.printStackTrace();
      }
      BaseLog.d(this.getClass(), "抓取链接详细内容结束================================================================");
      
      return content;
  }
   
  /**
   * 过滤所有以<开头以>结尾的标签
   */
  public static String filterHtml(String str) {   
      Pattern pattern = Pattern.compile("<([^>]*)>");   
      Matcher matcher = pattern.matcher(str);   
      StringBuffer sb = new StringBuffer();   
      boolean result = matcher.find();   
      while (result) {   
          matcher.appendReplacement(sb, "");   
          result = matcher.find();   
      }   
      matcher.appendTail(sb);   
      return sb.toString();   
  }
  
  /**
   * 抽取文本中的图片并下载到本地
   * @param text
   * @param access_path 图片访问地址前缀
   * @param location_path 图片存储路径
   * @return
   * @throws Exception
   */
  public String watchImg(String text) throws Exception{
    // 抓出图片地址 替换图片标示
    String regex = "<img[^>]*src=(\"([^\"]*)\"|\'([^\']*)\')[^>]*>";
    final Pattern pa = Pattern.compile(regex, Pattern.DOTALL);
    final Matcher ma = pa.matcher(text);
    
    while (ma.find()) {
        String imgUrl = ma.group(2);
        if(imgUrl.matches("(http|https)://.*")){
            // 获取文件格式后缀 组成本地文件保存路径
            String suffix = "."+imgUrl.replaceAll("^.*\\.", "");
            
            // 将匹配到的内容 替换为既定的表现形式
            String uploadImgurl = FileUtil.getFilePath(PREFIX,suffix);
            text = text.replace(ma.group(0), "<img src=\""+uploadImgurl+"\">");
            // 下载图片到既定位置 upload/doc/年月日/md5.jpg
            // 启动下载线程异步下载 
            GlobalConstant.uploadPool.execute(new FileDownLoadThread(imgUrl, GlobalConstant.PROJECT_WEBAPPS_PATH + uploadImgurl));
        }
    }
    return text;
  }
    
}
